{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "860c9e4b-dc7c-4f2e-8f60-96cccf61d43c",
   "metadata": {},
   "source": [
    "# OpenLayer\n",
    "## Evaluating RAG pipelines with Openlayer and Ragas\n",
    "\n",
    "[Openlayer](https://www.openlayer.com/) is an evaluation tool that fits into your development and production pipelines to help you ship high-quality models with confidence.\n",
    "\n",
    "This notebook should be used together with [this blog post](https://www.openlayer.com/blog/post/evaluating-rag-pipelines-with-ragas-and-openlayer)."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ad3ed0c-e495-4078-ab95-a70fa6322ab1",
   "metadata": {},
   "source": [
    "## Pre-requisites"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ded5103-b6ac-482e-9217-347f701333b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "git clone https://huggingface.co/datasets/explodinggradients/prompt-engineering-papers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58f0951f-5de9-4eca-8b0c-e77d5ac99bad",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"YOUR_OPENAI_API_KEY_HERE\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93b95703-0826-47b2-8b0b-e0f982b1e170",
   "metadata": {},
   "source": [
    "## Synthetic test data generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69cfc916-148a-4608-8eac-b75cc988b228",
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index import SimpleDirectoryReader\n",
    "from ragas.testset.generator import TestsetGenerator\n",
    "from ragas.testset.evolutions import simple, reasoning, multi_context\n",
    "\n",
    "# load documents\n",
    "dir_path = \"./prompt-engineering-papers\"\n",
    "reader = SimpleDirectoryReader(dir_path, num_files_limit=2)\n",
    "documents = reader.load_data()\n",
    "\n",
    "# generator with openai models\n",
    "generator = TestsetGenerator.with_openai()\n",
    "\n",
    "# set question type distribution\n",
    "distribution = {simple: 0.5, reasoning: 0.25, multi_context: 0.25}\n",
    "\n",
    "# generate testset\n",
    "testset = generator.generate_with_llamaindex_docs(\n",
    "    documents, test_size=10, distributions=distribution\n",
    ")\n",
    "test_df = testset.to_pandas()\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c802981-892e-4fed-bb73-dede5540fc6c",
   "metadata": {},
   "source": [
    "## Building RAG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72167cb6-bd8a-4d8b-a14c-142235f2ebe0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import nest_asyncio\n",
    "from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContext\n",
    "from llama_index.embeddings import OpenAIEmbedding\n",
    "\n",
    "\n",
    "nest_asyncio.apply()\n",
    "\n",
    "\n",
    "def build_query_engine(documents):\n",
    "    vector_index = VectorStoreIndex.from_documents(\n",
    "        documents,\n",
    "        service_context=ServiceContext.from_defaults(chunk_size=512),\n",
    "        embed_model=OpenAIEmbedding(),\n",
    "    )\n",
    "\n",
    "    query_engine = vector_index.as_query_engine(similarity_top_k=2)\n",
    "    return query_engine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5e47e5b-fa1a-4f07-b4a4-7493b1d58cc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "query_engine = build_query_engine(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6469b8ef-f9a3-4fb0-887a-0b70bce59dc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_single_response(query_engine, question):\n",
    "    response = query_engine.query(question)\n",
    "    return {\n",
    "        \"answer\": response.response,\n",
    "        \"contexts\": [c.node.get_content() for c in response.source_nodes],\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2123caed-a573-4e4e-bb60-41c15de6705f",
   "metadata": {},
   "outputs": [],
   "source": [
    "question = \"What are some strategies proposed to enhance the in-context learning capability of language models?\"\n",
    "generate_single_response(query_engine, question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c88035b-3383-44a6-bd8a-08a172f11a36",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import Dataset\n",
    "\n",
    "\n",
    "def generate_ragas_dataset(query_engine, test_df):\n",
    "    test_questions = test_df[\"question\"].values\n",
    "    responses = [generate_single_response(query_engine, q) for q in test_questions]\n",
    "\n",
    "    dataset_dict = {\n",
    "        \"question\": test_questions,\n",
    "        \"answer\": [response[\"answer\"] for response in responses],\n",
    "        \"contexts\": [response[\"contexts\"] for response in responses],\n",
    "        \"ground_truth\": test_df[\"ground_truth\"].values.tolist(),\n",
    "    }\n",
    "    ds = Dataset.from_dict(dataset_dict)\n",
    "    return ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "437368a5-3819-4ae1-b825-ad95664206ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "ragas_dataset = generate_ragas_dataset(query_engine, test_df)\n",
    "ragas_df = ragas_dataset.to_pandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10702a1e-276d-45f9-9d81-2be1bd98ce3d",
   "metadata": {},
   "source": [
    "## Commit to Openlayer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ced5f583-b849-4aae-8397-2bd9006bb69f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from openlayer.tasks import TaskType\n",
    "\n",
    "client = openlayer.OpenlayerClient(\"YOUR_OPENLAYER_API_KEY_HERE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15c6af02-c9bc-4368-82a1-43cf849446d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "project = client.create_project(\n",
    "    name=\"My-Rag-Project\",\n",
    "    task_type=TaskType.LLM,\n",
    "    description=\"Evaluating an LLM used for product development.\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "419f1392-4c44-4856-af5f-1bd04de1de7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_dataset_config = {\n",
    "    \"contextColumnName\": \"contexts\",\n",
    "    \"questionColumnName\": \"question\",\n",
    "    \"inputVariableNames\": [\"question\"],\n",
    "    \"label\": \"validation\",\n",
    "    \"outputColumnName\": \"answer\",\n",
    "    \"groundTruthColumnName\": \"ground_truth\",\n",
    "}\n",
    "project.add_dataframe(\n",
    "    dataset_df=ragas_df,\n",
    "    dataset_config=validation_dataset_config,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31c51305-2808-4cae-85c2-b261ca0d98c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_config = {\n",
    "    \"inputVariableNames\": [\"question\"],\n",
    "    \"modelType\": \"shell\",\n",
    "    \"metadata\": {\"top_k\": 2, \"chunk_size\": 512, \"embeddings\": \"OpenAI\"},\n",
    "}\n",
    "project.add_model(model_config=model_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "471643ba-5e5d-4500-9745-f0c355f744a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "project.commit(\"Initial commit!\")\n",
    "project.push()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b602dbbc-cc60-48b5-9bab-ae684c61cbff",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
